﻿using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Web;

namespace WI_E2_Parser
{
    class Crawler
    {
        public List<string> resultTokens;

        public string getExtension(string path)
        { 
            if(path.Contains("."))
            {
                string result = path.Substring(path.LastIndexOf("."), path.Length - path.LastIndexOf("."));
                return result;
            }
            else
            {
                return "";
            }
        }

        public bool allowedToVisit(string link, List<string> robotIgnores)
        {
            if (robotIgnores.Contains("/"))
            {
                return false;
            }

            foreach (string s in robotIgnores)
            {
                if (link.Contains(s))
                {
                    return false;
                }
            }
            return true;
        }

        public string normalizeLink(string link)
        {
            if (link.Contains('#'))
            {
                link = link.Substring(0, link.IndexOf('#'));
            }

            if (link[link.Length-1] == '/')
            {
                link = link.Remove(link.Length-1);
            }

            link = link.Replace("http://www.", "http://");

            return link;
        }

        internal static string RemoveUnwantedTags(string data)
        {
            var document = new HtmlDocument();
            document.LoadHtml(data);

            document.DocumentNode.Descendants()
                .Where(n => n.Name == "script" || n.Name == "style")
                .ToList()
                .ForEach(n => n.Remove());

            var nodes = new Queue<HtmlNode>(document.DocumentNode.SelectNodes("./*|./text()"));
            while (nodes.Count > 0)
            {
                var node = nodes.Dequeue();
                var parentNode = node.ParentNode;

                if (node.Name != "strong" && node.Name != "em" && node.Name != "u" && node.Name != "#text")
                {

                    var childNodes = node.SelectNodes("./*|./text()");

                    if (childNodes != null)
                    {
                        foreach (var child in childNodes)
                        {
                            nodes.Enqueue(child);
                            parentNode.InsertBefore(child, node);
                        }
                    }
                    parentNode.RemoveChild(node);
                }
            }

            return document.DocumentNode.InnerHtml;
        }

        public List<string> findLinks(string URL, List<string> ignoreExtensions, List<string> robotIgnores)
        {
            if (!allowedToVisit(URL, robotIgnores))
            {
                return new List<string>();
            }

            List<string> foundLinks = new List<string>();
            resultTokens = new List<string>();

            WebClient web = new WebClient();
            string page = "";

            try
            {
                page = web.DownloadString(URL);
                Console.WriteLine(" Crawling: " + URL);
            }
            catch (WebException e)
            { 
            }

            if (page.Length <= 0) {
                return new List<string>();
            }

            string strippedPage = RemoveUnwantedTags(page.ToLower());

            string replaceWith = " ";
            string removedBreaks = strippedPage.Replace("\r\n", replaceWith).Replace("\n", replaceWith).Replace("\r", replaceWith);
            removedBreaks = removedBreaks.Replace("</form>", "");
            removedBreaks = removedBreaks.Replace("\t", "");

            Tokenizer tok = new Tokenizer();
            
            Stemmer s = new Stemmer();



            foreach (string token in tok.tokenize(removedBreaks))
            {
                if(Regex.IsMatch(token, @"^[a-zA-Z]+$"))
                { 
                    this.resultTokens.Add(s.stem(token));
                }
            }
            
            Match m;
            string HRefPattern = "href\\s*=\\s*(?:[\"'](?<1>[^\"']*)[\"']|(?<1>\\S+))";

            try
            {
                m = Regex.Match(page, HRefPattern);
                while (m.Success)
                {
                    string foundLink = m.Groups[1].ToString();

                    //If link does not contain the domain, insert it
                    if (!foundLink.Contains("http"))
                    {
                        foundLink = URL + foundLink;
                    }

                    if (!ignoreExtensions.Contains(getExtension(foundLink)))
                    {
                        foundLink = normalizeLink(foundLink);
                        //If list of found links does not already contain the link, add it
                        if (!foundLinks.Contains(foundLink))
                        {
                            if(!foundLink.Contains("mailto:") && !foundLink.Contains("javascript:"))
                            {
                                if (allowedToVisit(foundLink, robotIgnores))
                                {
                                    foundLinks.Add(HttpUtility.UrlDecode(foundLink));
                                }
                            }
                        }
                    }
                    
                    m = m.NextMatch();
                }
            }
            catch (RegexMatchTimeoutException)
            {
                //Ignore timeout
            }

            return foundLinks;
        }
    }
}
